package edu.hawaii.webspider;

import static java.util.Collections.max;
import static java.util.Collections.singletonMap;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import org.xml.sax.SAXException;
import com.meterware.httpunit.HttpUnitOptions;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebResponse;

/**
 * A simple web spider.
 * 
 * Created using WebSpiderExample as a starting point.
 * 
 * @author Brian Jaress
 */
public class WebSpider {

  /**
   * Name of the class logger.
   */
  public static final String LOG = "edu.hawaii.webspider.WebSpider";

  /**
   * Handles command-line arguments.
   * 
   * @param arg [-totallinks url max_pages] [-mostpopular url max_pages]
   * @throws Exception If problems occur.
   */
  public static void main(String[] arg) throws Exception {

    // code from Kevin English to suppress stack traces.
    HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);
    HttpUnitOptions.setScriptingEnabled(false);

    // set up logging
    boolean logging = arg[arg.length - 1].equals("-logging");
    Logger.getLogger(LOG).setUseParentHandlers(logging);
    Logger.getLogger(WebIterator.LOG).setUseParentHandlers(logging);

    WebSpider spider = new WebSpider();
    for (int i = 0; i < arg.length; i++) {
      if (arg[i].equals("-totallinks")) {
        System.out.println("Total Links: "
            + spider.totalLinks(arg[i + 1], Integer
                .parseInt(arg[i + 2])));
      }
      if (arg[i].equals("-mostpopular")) {
        Map.Entry<String, Integer> winner = spider.mostPopular(
            arg[i + 1], Integer.parseInt(arg[i + 2]));
        System.out.println("Most Popular: " + winner.getKey()
            + " with " + winner.getValue());
      }
    }
  }

  private final Logger logger;

  /**
   * Create a WebSpider.
   */
  public WebSpider() {
    logger = Logger.getLogger(LOG);
  }

  /**
   * Count links seen while spidering pages.
   * 
   * @param url starting page URL
   * @param maxPages max number of pages to look at
   * @throws Exception on any error
   * 
   * @return the number of links
   */
  public int totalLinks(String url, int maxPages) throws Exception {
    WebIterator wi = new WebIterator();
    wi.setStartPage(url);
    int pages = 0;
    int links = 0;

    for (WebResponse response : wi) {
      if (pages >= maxPages) {
        break;
      }
      pages++;
      try {
        links += response.getLinks().length;
        logger.info(response.getLinks().length + " links in "
            + response.getURL());
      }
      catch (Exception e) {
        // if there are no links, there are no links
        continue;
      }
    }
    return links;
  }

  /**
   * Find the most popular page amongst the spidered pages.
   * 
   * A page's popularity is the number of pages which link to that page.
   * 
   * @param url starting page
   * @param maxPages maximum number of pages to spider
   * @return the URL of the most popular page and its score as a map
   * entry
   * @throws SAXException if it cannot parse the starting page
   * @throws IOException if it cannot get the starting page
   * @throws MalformedURLException if the starting page has a bad url
   */
  public Map.Entry<String, Integer> mostPopular(String url, int maxPages)
      throws MalformedURLException, IOException, SAXException {
    WebIterator wi = new WebIterator();
    wi.setStartPage(url);
    Set<String> seen = new HashSet<String>();
    final Map<String, Integer> score = new HashMap<String, Integer>();
    for (WebResponse response : wi) {
      if (seen.size() >= maxPages) {
        break;
      }
      seen.add(response.getURL().toString());
      WebLink[] links;
      try {
        links = response.getLinks();
        logger.info(links.length + " links in " + response.getURL());
      }
      catch (Exception e) {
        // if there are no links, we move on to the next page
        continue;
      }
      // don't count duplicate links on the same page
      Set<String> currentPageLinks = new HashSet<String>();
      for (WebLink link : links) {
        try {
          currentPageLinks.add(link.getRequest().getURL().toString());
        }
        catch (Exception e) {
          // if the link is bad, we move on to the next link
          continue;
        }
      }
      for (String linkUrl : currentPageLinks) {
        if (score.containsKey(linkUrl)) {
          score.put(linkUrl, score.get(linkUrl) + 1);
        }
        else {
          score.put(linkUrl, 1);
        }
      }
    }
    String winner = max(seen, new Comparator<String>() {
      public int compare(String a, String b) {
        Integer aInt = score.containsKey(a) ? score.get(a) : 0;
        Integer bInt = score.containsKey(b) ? score.get(b) : 0;
        return aInt.compareTo(bInt);
      }
    });
    // well, this is a hack. There must be a better way.
    Map<String, Integer> dummy = singletonMap(winner, score.get(winner));
    return dummy.entrySet().iterator().next();
  }
}
